import os
import pandas as pd

def get_reachable_domains(n:int, probed_file:str, domains_file:str, reachable_domains_file:str) -> str:
    """Keep reachable domains.

    If a domain's URL is reachable, its final status code should be 2xx.

    Args:
        n: Read top n line of input file.
        probed_file: Probed file path.
        domains_file: Domains file.
        reachable_domains_file: Output file path.

    Returns:
        reachable_domains_file: Reachable URLs file.
    """
    reachable_domains = []

    # Read probed file
    df = pd.read_csv(probed_file, encoding_errors="ignore")

    # Preserve links with status code 2xx
    df = df[(df["status-code"]>=200) & (df["status-code"]<300)]["url"]

    # Remove prefix ("https://") and postfix (":443"), save to a set
    domains = set(df.apply(lambda x: x[8:-4]).tolist())

    # Keep the original order
    with open(domains_file) as f_in:
        for _ in range(n):
            domain = f_in.readline().strip()
            if domain in domains:
                reachable_domains.append(domain)
    
    with open(reachable_domains_file, "w") as f_out:
        for domain in reachable_domains:
            f_out.write(domain + "\n")

    return reachable_domains_file

def generate_urls_file(n:int, input_file:str) -> str:
    """Generate a urls file based on input file:

    Args:
        n: Read top n line of input file.
        input_file: Input file path.

    Returns:
        output_file: Output file path.
    """
    output_file = "urls.txt"

    with open(output_file, "w") as f_out:
        with open(input_file) as f_in:
            for _ in range(n):     
                url = "https://" + f_in.readline().strip()
                f_out.write(url + "\n")

    return output_file

def main(config:dict):
    urls_file = generate_urls_file(config["TOP"], config["DOMAINS_FILE"])

    cmd = "{} -l {} -timeout {} -o {} -csv -s -sc -fr".format(
        config["HTTPX_PATH"],
        urls_file,
        config["TIMEOUT"],
        config["PROBED_FILE"]
    )

    os.system(cmd)
    os.remove(urls_file)

    get_reachable_domains(
        config["TOP"],
        config["PROBED_FILE"],
        config["DOMAINS_FILE"],
        config["REACHABLE_FILE"]
    )

if __name__ == "__main__":

    config = {
        "TOP": 10000,
        "HTTPX_PATH": "httpx/httpx",
        "DOMAINS_FILE": "chinaz-ranking-20220206.txt",
        "PROBED_FILE": "temp-probed.csv",
        "REACHABLE_FILE": "chinaz-ranking-reachable-20220206.txt",
        "DOMAINS_FILE_WITH_HTTPS_ENABLE": "chinaz-ranking-https-20220206.txt",
        "TIMEOUT": 10
    }

    main(config)